{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "12066209-f6f4-43bb-b664-b905079ba06b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=\"/pipeline/#/experiments/details/52b34c3f-6851-4467-8c67-9b93106fb162\" target=\"_blank\" >Experiment details</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a href=\"/pipeline/#/runs/details/c7fd55ff-5926-4ca7-84fd-9b7a33cc0e34\" target=\"_blank\" >Run details</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import kfp\n",
    "from kfp.v2 import dsl\n",
    "from kfp.v2.dsl import (\n",
    "    component,\n",
    "    Input,\n",
    "    Output,\n",
    "    Artifact,\n",
    "    Dataset,\n",
    "    ClassificationMetrics,\n",
    "    Metrics,\n",
    "    HTML,\n",
    "    Markdown\n",
    ")\n",
    "\n",
    "from typing import NamedTuple\n",
    "@component(\n",
    "    packages_to_install=[],\n",
    "    base_image=\"python:3.8\"\n",
    ")\n",
    "def get_data_batch() -> NamedTuple('MyFunctionOutputs', [('output_name_1', str), ('output_name_2', str)]):\n",
    "    print(\"getting data\")\n",
    "\n",
    "    # Exports a sample tensorboard:\n",
    "    metadata = {\n",
    "      'outputs' : [{\n",
    "        'type': 'tensorboard',\n",
    "        'source': 'gs://ml-pipeline-dataset/tensorboard-train',\n",
    "      }]\n",
    "    }\n",
    "\n",
    "    # Exports two sample metrics:\n",
    "    metrics = {\n",
    "      'metrics': [{\n",
    "          'name': 'quotient',\n",
    "          'numberValue':  float(2),\n",
    "        },{\n",
    "          'name': 'remainder',\n",
    "          'numberValue':  float(3),\n",
    "        }]}\n",
    "\n",
    "    from collections import namedtuple\n",
    "    import json\n",
    "    divmod_output = namedtuple('MyFunctionOutputs', ['output_name_1', 'output_name_2'])\n",
    "    return divmod_output(\"test\", \"test2\")\n",
    "\n",
    "@component(\n",
    "    packages_to_install=[\"sklearn\"],\n",
    "    base_image=\"python:3.8\"\n",
    ")\n",
    "def get_latest_data(metrics: Output[ClassificationMetrics]):\n",
    "    print(\"Getting latest data\")\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    \n",
    "    train_y = [1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0]\n",
    "    predictions = [1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0]\n",
    "    \n",
    "    metrics.log_confusion_matrix(\n",
    "        ['Setosa', 'Versicolour'],\n",
    "        confusion_matrix(train_y, predictions).tolist() # .tolist() to convert np array to list.\n",
    "    )\n",
    "\n",
    "\n",
    "@component(\n",
    "    packages_to_install=[],\n",
    "    base_image=\"public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full:v1.4\"\n",
    ")\n",
    "def reshape_data():\n",
    "    print(\"reshaping data\")\n",
    "\n",
    "\n",
    "@component(\n",
    "    packages_to_install=[],\n",
    "    base_image=\"public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-tensorflow-full:v1.4\"\n",
    ")\n",
    "def model_building(no_epochs:int):\n",
    "    print(\"model building\")\n",
    "    print(no_epochs)\n",
    "    print(type(no_epochs))\n",
    "    \n",
    "\n",
    "@dsl.pipeline(\n",
    "    name='output-test',\n",
    "    description='test outputs'\n",
    ")\n",
    "def output_test(no_epochs:int):\n",
    "    step1_1 = get_data_batch()\n",
    "    step1_2 = get_latest_data()\n",
    "    \n",
    "    step2 = reshape_data()\n",
    "    step2.after(step1_1)\n",
    "    step2.after(step1_2)\n",
    "    \n",
    "    step3 = model_building(no_epochs)\n",
    "    step3.after(step2)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    client = kfp.Client()\n",
    "\n",
    "    arguments = {\n",
    "        \"no_epochs\" : 3\n",
    "    }\n",
    "\n",
    "    run_directly = 1\n",
    "    \n",
    "    if (run_directly == 1):\n",
    "        client.create_run_from_pipeline_func(output_test,arguments=arguments,experiment_name=\"test\",mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE)\n",
    "    else:\n",
    "        kfp.compiler.Compiler(mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE).compile(pipeline_func=output_test,package_path='output_test.yaml')\n",
    "        client.upload_pipeline_version(pipeline_package_path='output_test.yaml',pipeline_version_name=\"0.3\",pipeline_name=\"pipeline test\",description=\"just for testing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "433fe12c-6fa4-49e8-90b9-78acf05ba9b6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
